package com.bblocks.common.data.support;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import com.bblocks.area.AreaVo;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.collect.Lists;
import com.pig4cloud.pig.common.core.constant.CacheConstants;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.RedisTemplate;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

import java.util.List;
import java.util.concurrent.TimeUnit;

/**
 * <p>Project: bblocks-pro - AreaSpiderWithRedisCache</p>
 * <p>描述：爬取省市区街道社区（5级）-redis缓存以优化提取速度（二次提取速度）</p>
 *
 * @Author Zhao [125043150@qq.com]
 * @Date 2024/3/18 17:39
 * @Version 1.0
 * @since 8
 */
@Slf4j
public class AreaSpiderWithRedisCache extends AreaSpider {

        private List<AreaVo> result;
        private RedisTemplate redisTemplate;

        /**
         *
         * @param year
         * @param containLv
         * @param list
         * @param redisTemplate 允许为空
         */
        public AreaSpiderWithRedisCache(int year, int containLv, List<AreaVo> list
                , RedisTemplate redisTemplate) {
            super(year, containLv);

            this.result = list;
            this.redisTemplate = redisTemplate;
        }

        protected boolean addFollowsUrl(Page page, AreaVo parent, int targetLv) {
            String newUrl = parent.getChildUrl();
            if(StrUtil.isEmpty(newUrl))
                return false;
            List<AreaVo> areas = (List<AreaVo>)redisTemplate.opsForValue().get(CacheConstants.CRAWLING+":area:"+newUrl);
            if(CollUtil.isEmpty(areas)){
                page.addTargetRequest(newUrl);
                log.info("即将采集地址:{}" , newUrl);
                return true;
            }else {
                /**
                 * 子集 添加采集
                 */
                if(targetLv < getContainLv()){
                    areas.forEach(a->{
                        if(StrUtil.isNotEmpty(a.getChildUrl())){
                            addFollowsUrl(page,a,targetLv + 1);
                        }
                    });
                }
                this.result.addAll(areas);

                parent.setChildState(1);
                return false;
            }
        }


        public static void main(String[] args) {
            long b = System.currentTimeMillis();
            List<AreaVo> result = Lists.newArrayList();
            AreaSpiderWithCache.LogPipeline logPipeline = new AreaSpiderWithCache.LogPipeline(result);
            Spider.create(new AreaSpiderWithCache(2022, 1,result))
                    .addUrl(SPIDER_URL + "index.html")
                    .addPipeline(logPipeline)
                    .thread(2)
                    .run();

            log.info("采集到1:{}-{}-{}",logPipeline.getFailNum(),result.size(),(System.currentTimeMillis() - b));

            b = System.currentTimeMillis();
            result.clear();
            logPipeline = new AreaSpiderWithCache.LogPipeline(result);
            Spider.create(new AreaSpiderWithCache(2022, 1,result))
                    .addUrl(SPIDER_URL + "index.html")
                    .addPipeline(logPipeline)
                    .thread(2)
                    .run();
            log.info("采集到1:{}-{}-{}",logPipeline.getFailNum(),result.size(),(System.currentTimeMillis() - b));

        }

        @Data
        public static class LogPipeline implements Pipeline {

            private RedisTemplate redisTemplate;

            private List<AreaVo> result;
            /**
             * 缓存时间-分钟
             */
            private int cacheTime;
            /**
             * 失败次数 0=成功，
             */
            private int failNum  ;

            /**
             *
             * @param result
             * @param cacheTime 缓存时间-分钟
             * @param redisTemplate
             */
            public LogPipeline(List<AreaVo> result,int cacheTime,RedisTemplate redisTemplate ){
                this.result = result;
                this.failNum = 0;
                this.redisTemplate = redisTemplate;
                this.cacheTime = cacheTime;
            }

            @Override
            public void process(ResultItems resultItems, Task task) {
                String url = resultItems.getRequest().getUrl();
                List<AreaVo> areas = resultItems.get("area");
                log.info("地区总数：{}" , areas.size());


                if (areas.isEmpty()) {//url采集失败
                    log.info(url + " 此页面未爬取数据,请稍后重试!");
                    failNum ++;
                } else {
                    areas.stream().forEach(a -> {
                        log.info("{}" , a);
                    });
                    //缓存已采集数据
                    redisTemplate.opsForValue().set(CacheConstants.CRAWLING+":area:"+url,areas,cacheTime, TimeUnit.MINUTES);

                    this.result.addAll(areas);
                }
            }
        }

}
